Index

_

__enter__() (spark_pipeline_framework.progress_logger.progress_log_metric.ProgressLogMetric method) (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) __exit__() (spark_pipeline_framework.progress_logger.progress_log_metric.ProgressLogMetric method) (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) __getitem__() (spark_pipeline_framework.utilities.parameter_dict.ParameterDict method) __mlflow_clean_param_value() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) __mlflow_clean_string() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) __str__() (spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner.FrameworkMappingLoader method) (spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer.FrameworkSqlTransformer method) (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) _check_validation() (spark_pipeline_framework.pipelines.v2.framework_pipeline.FrameworkPipeline method) _set() (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) _setDefault() (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) _transform() (spark_pipeline_framework.pipelines.framework_param_pipeline.FrameworkParamPipeline method) (spark_pipeline_framework.pipelines.framework_pipeline.FrameworkPipeline method) (spark_pipeline_framework.pipelines.v2.framework_pipeline.FrameworkPipeline method) (spark_pipeline_framework.proxy_generator.proxy_base.ProxyBase method) (spark_pipeline_framework.proxy_generator.python_proxy_base.PythonProxyBase method) (spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter.FrameworkBaseExporter method) (spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer.FrameworkDataFrameAnalyzer method) (spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner.FrameworkDBQueryRunner method) (spark_pipeline_framework.transformers.framework_drop_duplicates_transformer.v1.framework_drop_duplicates_transformer.FrameworkDropDuplicatesTransformer method) (spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer.v1.framework_drop_rows_with_null_transformer.FrameworkDropRowsWithNullTransformer method) (spark_pipeline_framework.transformers.framework_file_downloader.v1.framework_file_downloader.FrameworkFileDownloader method) (spark_pipeline_framework.transformers.framework_fill_na_transformer.v1.framework_fill_na_transformer.FrameworkFillNaTransformer method) (spark_pipeline_framework.transformers.framework_filter_by_column.v1.framework_filter_by_column_transformer.FrameworkFilterByColumnTransformer method) (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.FrameworkFixedWidthLoader method) (spark_pipeline_framework.transformers.framework_if_else_transformer.v1.framework_if_else_transformer.FrameworkIfElseTransformer method) (spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader.FrameworkJdbcReader method) (spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter.FrameworkJsonExporter method) (spark_pipeline_framework.transformers.framework_json_splitter.v1.framework_json_splitter.FrameworkJsonSplitter method) (spark_pipeline_framework.transformers.framework_json_to_jsonl_converter.v1.framework_json_to_jsonl_converter.FrameworkJsonToJsonlConverter method) (spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader.FrameworkKafkaReader method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) (spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner.FrameworkMappingLoader method) (spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader.FrameworkParquetLoader method) (spark_pipeline_framework.transformers.framework_partitioner.v1.framework_partitioner.FrameworkPartitioner method) (spark_pipeline_framework.transformers.framework_select_columns_transformer.v1.framework_select_columns_transformer.FrameworkSelectColumnsTransformer method) (spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer.FrameworkSqlTransformer method) (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) (spark_pipeline_framework.transformers.framework_url_loader.v1.framework_url_loader.FrameworkUrlLoader method) (spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer.FrameworkValidationTransformer method) (spark_pipeline_framework.transformers.framework_xml_loader.v1.framework_xml_loader.FrameworkXmlLoader method)

_validate() (spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer.FrameworkValidationTransformer method)

A

add_metadata_to_column() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers)

AutoMapperFunction (in module spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner) AutoMapperTypeOrList (in module spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner)

B

BaseSlackClient (class in spark_pipeline_framework.utilities.slack.base_slack_client)

C

check_if_file_is_jsonl() (in module spark_pipeline_framework.utilities.json_to_jsonl_converter) check_if_path_exists() (spark_pipeline_framework.utilities.file_downloader.FileDownloader static method) ClassHelpers (class in spark_pipeline_framework.utilities.class_helpers) column_name (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.ColumnSpec attribute) ColumnSpec (class in spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader) convert_json_to_jsonl() (in module spark_pipeline_framework.utilities.json_to_jsonl_converter) convert_to_row() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) copy() (spark_pipeline_framework.utilities.parameter_dict.ParameterDict method)

create_dataframe_from_dictionary() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) create_dataframe_from_json() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) create_empty_dataframe() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) create_steps() (in module spark_pipeline_framework.utilities.pipeline_helper) (spark_pipeline_framework.pipelines.framework_param_pipeline.FrameworkParamPipeline method) (spark_pipeline_framework.pipelines.framework_pipeline.FrameworkPipeline method) (spark_pipeline_framework.pipelines.v2.framework_pipeline.FrameworkPipeline method) create_view_from_dictionary() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers)

D

data_type (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.ColumnSpec attribute) diff_lists() (in module spark_pipeline_framework.utilities.list_utils)

download_files_from_url() (spark_pipeline_framework.utilities.file_downloader.FileDownloader method) download_files_locally() (spark_pipeline_framework.utilities.file_downloader.FileDownloader method) download_files_to_s3() (spark_pipeline_framework.utilities.file_downloader.FileDownloader method)

E

end_mlflow_run() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) EventLogger (class in spark_pipeline_framework.event_loggers.event_logger)

exception_summary() (spark_pipeline_framework.utilities.FriendlySparkException.FriendlySparkException static method) extract_zip_files() (spark_pipeline_framework.utilities.file_downloader.FileDownloader method)

F

FileDownloader (class in spark_pipeline_framework.utilities.file_downloader) FileReadModes (class in spark_pipeline_framework.utilities.file_modes) FileWriteModes (class in spark_pipeline_framework.utilities.file_modes) finalize() (spark_pipeline_framework.pipelines.framework_pipeline.FrameworkPipeline method) (spark_pipeline_framework.pipelines.v2.framework_pipeline.FrameworkPipeline method) fit() (spark_pipeline_framework.pipelines.framework_param_pipeline.FrameworkParamPipeline method) (spark_pipeline_framework.pipelines.framework_pipeline.FrameworkPipeline method) (spark_pipeline_framework.pipelines.v2.framework_pipeline.FrameworkPipeline method) (spark_pipeline_framework.proxy_generator.proxy_base.ProxyBase method) flatten() (in module spark_pipeline_framework.utilities.flattener) FrameworkBaseExporter (class in spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter) FrameworkCsvExporter (class in spark_pipeline_framework.transformers.framework_csv_exporter.v1.framework_csv_exporter) FrameworkCsvLoader (class in spark_pipeline_framework.transformers.framework_csv_loader.v1.framework_csv_loader) FrameworkDataFrameAnalyzer (class in spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer) FrameworkDBQueryRunner (class in spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner) FrameworkDropDuplicatesTransformer (class in spark_pipeline_framework.transformers.framework_drop_duplicates_transformer.v1.framework_drop_duplicates_transformer) FrameworkDropRowsWithNullTransformer (class in spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer.v1.framework_drop_rows_with_null_transformer) FrameworkFileDownloader (class in spark_pipeline_framework.transformers.framework_file_downloader.v1.framework_file_downloader) FrameworkFillNaTransformer (class in spark_pipeline_framework.transformers.framework_fill_na_transformer.v1.framework_fill_na_transformer) FrameworkFilterByColumnTransformer (class in spark_pipeline_framework.transformers.framework_filter_by_column.v1.framework_filter_by_column_transformer) FrameworkFixedWidthLoader (class in spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader) FrameworkIfElseTransformer (class in spark_pipeline_framework.transformers.framework_if_else_transformer.v1.framework_if_else_transformer)

FrameworkJdbcExporter (class in spark_pipeline_framework.transformers.framework_jdbc_exporter.v1.framework_jdbc_exporter) FrameworkJdbcReader (class in spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader) FrameworkJsonExporter (class in spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter) FrameworkJsonLoader (class in spark_pipeline_framework.transformers.framework_json_loader.v1.framework_json_loader) FrameworkJsonSplitter (class in spark_pipeline_framework.transformers.framework_json_splitter.v1.framework_json_splitter) FrameworkJsonToJsonlConverter (class in spark_pipeline_framework.transformers.framework_json_to_jsonl_converter.v1.framework_json_to_jsonl_converter) FrameworkKafkaReader (class in spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader) FrameworkLocalFileLoader (class in spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader) FrameworkMappingLoader (class in spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner) FrameworkMappingRunnerException FrameworkParamPipeline (class in spark_pipeline_framework.pipelines.framework_param_pipeline) FrameworkParquetExporter (class in spark_pipeline_framework.transformers.framework_parquet_exporter.v1.framework_parquet_exporter) FrameworkParquetLoader (class in spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader) FrameworkPartitioner (class in spark_pipeline_framework.transformers.framework_partitioner.v1.framework_partitioner) FrameworkPipeline (class in spark_pipeline_framework.pipelines.framework_pipeline) (class in spark_pipeline_framework.pipelines.v2.framework_pipeline) FrameworkSelectColumnsTransformer (class in spark_pipeline_framework.transformers.framework_select_columns_transformer.v1.framework_select_columns_transformer) FrameworkSqlTransformer (class in spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer) FrameworkTransformer (class in spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer) FrameworkUrlLoader (class in spark_pipeline_framework.transformers.framework_url_loader.v1.framework_url_loader) FrameworkValidationTransformer (class in spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer) FrameworkXmlLoader (class in spark_pipeline_framework.transformers.framework_xml_loader.v1.framework_xml_loader) FriendlySparkException

G

generate_proxies() (spark_pipeline_framework.proxy_generator.proxy_generator.ProxyGenerator static method) get_absolute_paths() (in module spark_pipeline_framework.utilities.file_helpers) get_dtype() (in module spark_pipeline_framework.transformers.framework_fill_na_transformer.v1.framework_fill_na_transformer) get_errortext() (spark_pipeline_framework.utilities.FriendlySparkException.FriendlySparkException static method) get_filename_from_url() (spark_pipeline_framework.utilities.file_downloader.FileDownloader static method) get_first_class_in_file() (spark_pipeline_framework.utilities.class_helpers.ClassHelpers static method) get_first_class_in_module() (in module spark_pipeline_framework.utilities.module_utilities) get_grafana_url() (spark_pipeline_framework.event_loggers.slack_logger.v1.slack_event_logger.SlackEventLogger method) get_logger() (in module spark_pipeline_framework.logger.yarn_logger) get_mapping() (spark_pipeline_framework.proxy_generator.translator_proxy_base.TranslatorProxyBase method) get_metadata_of_column() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) get_python_function_from_location() (in module spark_pipeline_framework.proxy_generator.python_transformer_helpers) get_python_mapping_transformer() (spark_pipeline_framework.proxy_generator.proxy_base.ProxyBase method) get_python_transformer() (spark_pipeline_framework.proxy_generator.proxy_base.ProxyBase method) get_python_transformer_from_location() (in module spark_pipeline_framework.proxy_generator.python_transformer_helpers) get_validation_df() (spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer.FrameworkValidationTransformer method) getAnalysisViewsPrefix() (spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer.FrameworkDataFrameAnalyzer method) getCacheTable() (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) getCleanColumnNames() (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) getColumn() (spark_pipeline_framework.transformers.framework_filter_by_column.v1.framework_filter_by_column_transformer.FrameworkFilterByColumnTransformer method) getColumnMapping() (spark_pipeline_framework.transformers.framework_fill_na_transformer.v1.framework_fill_na_transformer.FrameworkFillNaTransformer method) getColumns() (spark_pipeline_framework.transformers.framework_drop_duplicates_transformer.v1.framework_drop_duplicates_transformer.FrameworkDropDuplicatesTransformer method) (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.FrameworkFixedWidthLoader method) getColumnsToAnalyze() (spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer.FrameworkDataFrameAnalyzer method) getColumnsToCheck() (spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer.v1.framework_drop_rows_with_null_transformer.FrameworkDropRowsWithNullTransformer method) getColumnsToSkip() (spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer.FrameworkDataFrameAnalyzer method) getCreateFilePath() (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) getDb() (spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner.FrameworkDBQueryRunner method) getDelimiter() (spark_pipeline_framework.transformers.framework_csv_exporter.v1.framework_csv_exporter.FrameworkCsvExporter method) (spark_pipeline_framework.transformers.framework_csv_loader.v1.framework_csv_loader.FrameworkCsvLoader method) getDesiredPartitions() (spark_pipeline_framework.transformers.framework_partitioner.v1.framework_partitioner.FrameworkPartitioner method) getDownloadToPath() (spark_pipeline_framework.transformers.framework_file_downloader.v1.framework_file_downloader.FrameworkFileDownloader method) getDownloadUrls() (spark_pipeline_framework.transformers.framework_file_downloader.v1.framework_file_downloader.FrameworkFileDownloader method) getDriver() (spark_pipeline_framework.transformers.framework_jdbc_exporter.v1.framework_jdbc_exporter.FrameworkJdbcExporter method) (spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader.FrameworkJdbcReader method) getExtractZips() (spark_pipeline_framework.transformers.framework_file_downloader.v1.framework_file_downloader.FrameworkFileDownloader method) getFailOnValidation() (spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer.FrameworkValidationTransformer method) getFilePath() (spark_pipeline_framework.transformers.framework_csv_exporter.v1.framework_csv_exporter.FrameworkCsvExporter method) getFilepath() (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.FrameworkFixedWidthLoader method) getFilePath() (spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter.FrameworkJsonExporter method) (spark_pipeline_framework.transformers.framework_json_splitter.v1.framework_json_splitter.FrameworkJsonSplitter method) (spark_pipeline_framework.transformers.framework_json_to_jsonl_converter.v1.framework_json_to_jsonl_converter.FrameworkJsonToJsonlConverter method) getFilepath() (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) getFilePath() (spark_pipeline_framework.transformers.framework_parquet_exporter.v1.framework_parquet_exporter.FrameworkParquetExporter method) (spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader.FrameworkParquetLoader method) getFilepath() (spark_pipeline_framework.transformers.framework_xml_loader.v1.framework_xml_loader.FrameworkXmlLoader method) getFormat() (spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter.FrameworkBaseExporter method) (spark_pipeline_framework.transformers.framework_csv_exporter.v1.framework_csv_exporter.FrameworkCsvExporter method) (spark_pipeline_framework.transformers.framework_jdbc_exporter.v1.framework_jdbc_exporter.FrameworkJdbcExporter method) (spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader.FrameworkJdbcReader method) (spark_pipeline_framework.transformers.framework_parquet_exporter.v1.framework_parquet_exporter.FrameworkParquetExporter method) getHasHeader() (spark_pipeline_framework.transformers.framework_csv_loader.v1.framework_csv_loader.FrameworkCsvLoader method) (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.FrameworkFixedWidthLoader method) getHeader() (spark_pipeline_framework.transformers.framework_csv_exporter.v1.framework_csv_exporter.FrameworkCsvExporter method) getHost() (spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner.FrameworkDBQueryRunner method) getIncludeOnly() (spark_pipeline_framework.transformers.framework_filter_by_column.v1.framework_filter_by_column_transformer.FrameworkFilterByColumnTransformer method) getInferSchema() (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) getJdbcUrl() (spark_pipeline_framework.transformers.framework_jdbc_exporter.v1.framework_jdbc_exporter.FrameworkJdbcExporter method) (spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader.FrameworkJdbcReader method) getKafkaBrokers() (spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader.FrameworkKafkaReader method) getLimit() (spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter.FrameworkBaseExporter method) (spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter.FrameworkJsonExporter method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) (spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader.FrameworkParquetLoader method) getLogSql() (spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer.FrameworkSqlTransformer method) getMappingFileName() (spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer.FrameworkSqlTransformer method) getMappingFunction() (spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner.FrameworkMappingLoader method) getMaxSizePerFileInMb() (spark_pipeline_framework.transformers.framework_json_splitter.v1.framework_json_splitter.FrameworkJsonSplitter method) getMergeSchema() (spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader.FrameworkParquetLoader method)

getMethod() (spark_pipeline_framework.transformers.framework_url_loader.v1.framework_url_loader.FrameworkUrlLoader method) getMode() (spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter.FrameworkBaseExporter method) (spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter.FrameworkJsonExporter method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) (spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader.FrameworkParquetLoader method) getMultiline() (spark_pipeline_framework.transformers.framework_csv_loader.v1.framework_csv_loader.FrameworkCsvLoader method) getMultiLine() (spark_pipeline_framework.transformers.framework_json_loader.v1.framework_json_loader.FrameworkJsonLoader method) getName() (spark_pipeline_framework.proxy_generator.python_proxy_base.PythonProxyBase method) (spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter.FrameworkJsonExporter method) (spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader.FrameworkKafkaReader method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) getOptions() (spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter.FrameworkBaseExporter method) (spark_pipeline_framework.transformers.framework_csv_exporter.v1.framework_csv_exporter.FrameworkCsvExporter method) (spark_pipeline_framework.transformers.framework_jdbc_exporter.v1.framework_jdbc_exporter.FrameworkJdbcExporter method) (spark_pipeline_framework.transformers.framework_parquet_exporter.v1.framework_parquet_exporter.FrameworkParquetExporter method) getOutputFolder() (spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer.FrameworkDataFrameAnalyzer method) (spark_pipeline_framework.transformers.framework_json_splitter.v1.framework_json_splitter.FrameworkJsonSplitter method) (spark_pipeline_framework.transformers.framework_json_to_jsonl_converter.v1.framework_json_to_jsonl_converter.FrameworkJsonToJsonlConverter method) getParameters() (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) getParamKey() (spark_pipeline_framework.transformers.framework_file_downloader.v1.framework_file_downloader.FrameworkFileDownloader method) getPartitionBy() (spark_pipeline_framework.transformers.framework_partitioner.v1.framework_partitioner.FrameworkPartitioner method) getPassword() (spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner.FrameworkDBQueryRunner method) getPort() (spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner.FrameworkDBQueryRunner method) getPreviousCheckpointView() (spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader.FrameworkKafkaReader method) getProgressLogger() (spark_pipeline_framework.proxy_generator.python_proxy_base.PythonProxyBase method) (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) getQuery() (spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner.FrameworkDBQueryRunner method) (spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader.FrameworkJdbcReader method) getReaderFormat() (spark_pipeline_framework.transformers.framework_csv_loader.v1.framework_csv_loader.FrameworkCsvLoader method) (spark_pipeline_framework.transformers.framework_json_loader.v1.framework_json_loader.FrameworkJsonLoader method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) getReaderOptions() (spark_pipeline_framework.transformers.framework_csv_loader.v1.framework_csv_loader.FrameworkCsvLoader method) (spark_pipeline_framework.transformers.framework_json_loader.v1.framework_json_loader.FrameworkJsonLoader method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) getRelativeUrl() (spark_pipeline_framework.transformers.framework_url_loader.v1.framework_url_loader.FrameworkUrlLoader method) getRowTag() (spark_pipeline_framework.transformers.framework_xml_loader.v1.framework_xml_loader.FrameworkXmlLoader method) getSchema() (spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader.FrameworkKafkaReader method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) (spark_pipeline_framework.transformers.framework_xml_loader.v1.framework_xml_loader.FrameworkXmlLoader method) getServerUrl() (spark_pipeline_framework.transformers.framework_url_loader.v1.framework_url_loader.FrameworkUrlLoader method) getSql() (spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer.FrameworkSqlTransformer method) (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) getStartingOffset() (spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader.FrameworkKafkaReader method) getTable() (spark_pipeline_framework.transformers.framework_jdbc_exporter.v1.framework_jdbc_exporter.FrameworkJdbcExporter method) getTopic() (spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader.FrameworkKafkaReader method) getUsername() (spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner.FrameworkDBQueryRunner method) getUseSchemaFromView() (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) getUseSsl() (spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader.FrameworkKafkaReader method) getValidationQueries() (spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer.FrameworkValidationTransformer method) getValidationSourcePath() (spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer.FrameworkValidationTransformer method) getVerifyCountRemainsSame() (spark_pipeline_framework.transformers.framework_select_columns_transformer.v1.framework_select_columns_transformer.FrameworkSelectColumnsTransformer method) (spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer.FrameworkSqlTransformer method) getView() (spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter.FrameworkBaseExporter method) (spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer.FrameworkDataFrameAnalyzer method) (spark_pipeline_framework.transformers.framework_drop_duplicates_transformer.v1.framework_drop_duplicates_transformer.FrameworkDropDuplicatesTransformer method) (spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer.v1.framework_drop_rows_with_null_transformer.FrameworkDropRowsWithNullTransformer method) (spark_pipeline_framework.transformers.framework_fill_na_transformer.v1.framework_fill_na_transformer.FrameworkFillNaTransformer method) (spark_pipeline_framework.transformers.framework_filter_by_column.v1.framework_filter_by_column_transformer.FrameworkFilterByColumnTransformer method) (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.FrameworkFixedWidthLoader method) (spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader.FrameworkJdbcReader method) (spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter.FrameworkJsonExporter method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) (spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner.FrameworkMappingLoader method) (spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader.FrameworkParquetLoader method) (spark_pipeline_framework.transformers.framework_partitioner.v1.framework_partitioner.FrameworkPartitioner method) (spark_pipeline_framework.transformers.framework_select_columns_transformer.v1.framework_select_columns_transformer.FrameworkSelectColumnsTransformer method) (spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer.FrameworkSqlTransformer method) (spark_pipeline_framework.transformers.framework_xml_loader.v1.framework_xml_loader.FrameworkXmlLoader method) getViews() (spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner.FrameworkMappingLoader method)

H

http_error_default() (spark_pipeline_framework.utilities.file_downloader.ThrowOnErrorOpener method)

I

instantiate_class_with_parameters() (spark_pipeline_framework.utilities.class_helpers.ClassHelpers static method)

isfile() (in module spark_pipeline_framework.utilities.file_helpers)

L

length (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.ColumnSpec attribute) listdir() (in module spark_pipeline_framework.utilities.file_helpers) load_all_modules_from_dir() (in module spark_pipeline_framework.proxy_generator.python_transformer_helpers) log_artifact() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) log_event() (spark_pipeline_framework.event_loggers.event_logger.EventLogger method) (spark_pipeline_framework.event_loggers.slack_logger.v1.slack_event_logger.SlackEventLogger method) (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) log_exception() (spark_pipeline_framework.event_loggers.event_logger.EventLogger method) (spark_pipeline_framework.event_loggers.slack_logger.v1.slack_event_logger.SlackEventLogger method) (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method)

log_metric() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) log_param() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) log_params() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) log_progress_event() (spark_pipeline_framework.event_loggers.event_logger.EventLogger method) (spark_pipeline_framework.event_loggers.slack_logger.v1.slack_event_logger.SlackEventLogger method) (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) logger (in module spark_pipeline_framework.utilities.slack.slack_client_native)

M

main() (in module spark_pipeline_framework.proxy_generator.generate_proxies) MlFlowConfig (class in spark_pipeline_framework.progress_logger.progress_logger) MODE_APPEND (spark_pipeline_framework.utilities.file_modes.FileWriteModes attribute) MODE_CHOICES (spark_pipeline_framework.utilities.file_modes.FileReadModes attribute) (spark_pipeline_framework.utilities.file_modes.FileWriteModes attribute) MODE_DROP_MALFORMED (spark_pipeline_framework.utilities.file_modes.FileReadModes attribute) MODE_ERROR (spark_pipeline_framework.utilities.file_modes.FileWriteModes attribute) MODE_FAIL_FAST (spark_pipeline_framework.utilities.file_modes.FileReadModes attribute) MODE_IGNORE (spark_pipeline_framework.utilities.file_modes.FileWriteModes attribute) MODE_OVERWRITE (spark_pipeline_framework.utilities.file_modes.FileWriteModes attribute) MODE_PERMISSIVE (spark_pipeline_framework.utilities.file_modes.FileReadModes attribute) module spark_pipeline_framework spark_pipeline_framework.event_loggers spark_pipeline_framework.event_loggers.event_logger spark_pipeline_framework.event_loggers.slack_logger spark_pipeline_framework.event_loggers.slack_logger.v1 spark_pipeline_framework.event_loggers.slack_logger.v1.slack_event_logger spark_pipeline_framework.logger spark_pipeline_framework.logger.yarn_logger spark_pipeline_framework.pipelines spark_pipeline_framework.pipelines.framework_param_pipeline spark_pipeline_framework.pipelines.framework_pipeline spark_pipeline_framework.pipelines.v2 spark_pipeline_framework.pipelines.v2.framework_pipeline spark_pipeline_framework.progress_logger spark_pipeline_framework.progress_logger.progress_log_metric spark_pipeline_framework.progress_logger.progress_logger spark_pipeline_framework.proxy_generator spark_pipeline_framework.proxy_generator.generate_proxies spark_pipeline_framework.proxy_generator.proxy_base spark_pipeline_framework.proxy_generator.proxy_generator spark_pipeline_framework.proxy_generator.python_proxy_base spark_pipeline_framework.proxy_generator.python_transformer_helpers spark_pipeline_framework.proxy_generator.translator_proxy_base spark_pipeline_framework.transformers spark_pipeline_framework.transformers.framework_base_exporter spark_pipeline_framework.transformers.framework_base_exporter.v1 spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter spark_pipeline_framework.transformers.framework_csv_exporter spark_pipeline_framework.transformers.framework_csv_exporter.v1 spark_pipeline_framework.transformers.framework_csv_exporter.v1.framework_csv_exporter spark_pipeline_framework.transformers.framework_csv_loader spark_pipeline_framework.transformers.framework_csv_loader.v1 spark_pipeline_framework.transformers.framework_csv_loader.v1.framework_csv_loader spark_pipeline_framework.transformers.framework_data_frame_analyzer spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1 spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer spark_pipeline_framework.transformers.framework_db_query_runner spark_pipeline_framework.transformers.framework_db_query_runner.v1 spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner spark_pipeline_framework.transformers.framework_drop_duplicates_transformer spark_pipeline_framework.transformers.framework_drop_duplicates_transformer.v1 spark_pipeline_framework.transformers.framework_drop_duplicates_transformer.v1.framework_drop_duplicates_transformer spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer.v1 spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer.v1.framework_drop_rows_with_null_transformer spark_pipeline_framework.transformers.framework_file_downloader spark_pipeline_framework.transformers.framework_file_downloader.v1 spark_pipeline_framework.transformers.framework_file_downloader.v1.framework_file_downloader spark_pipeline_framework.transformers.framework_fill_na_transformer spark_pipeline_framework.transformers.framework_fill_na_transformer.v1 spark_pipeline_framework.transformers.framework_fill_na_transformer.v1.framework_fill_na_transformer spark_pipeline_framework.transformers.framework_filter_by_column spark_pipeline_framework.transformers.framework_filter_by_column.v1 spark_pipeline_framework.transformers.framework_filter_by_column.v1.framework_filter_by_column_transformer spark_pipeline_framework.transformers.framework_fixed_width_loader spark_pipeline_framework.transformers.framework_fixed_width_loader.v1 spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader spark_pipeline_framework.transformers.framework_if_else_transformer spark_pipeline_framework.transformers.framework_if_else_transformer.v1 spark_pipeline_framework.transformers.framework_if_else_transformer.v1.framework_if_else_transformer spark_pipeline_framework.transformers.framework_jdbc_exporter spark_pipeline_framework.transformers.framework_jdbc_exporter.v1 spark_pipeline_framework.transformers.framework_jdbc_exporter.v1.framework_jdbc_exporter spark_pipeline_framework.transformers.framework_jdbc_reader spark_pipeline_framework.transformers.framework_jdbc_reader.v1 spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader spark_pipeline_framework.transformers.framework_json_exporter spark_pipeline_framework.transformers.framework_json_exporter.v1 spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter spark_pipeline_framework.transformers.framework_json_loader spark_pipeline_framework.transformers.framework_json_loader.v1 spark_pipeline_framework.transformers.framework_json_loader.v1.framework_json_loader spark_pipeline_framework.transformers.framework_json_splitter spark_pipeline_framework.transformers.framework_json_splitter.v1 spark_pipeline_framework.transformers.framework_json_splitter.v1.framework_json_splitter spark_pipeline_framework.transformers.framework_json_to_jsonl_converter spark_pipeline_framework.transformers.framework_json_to_jsonl_converter.v1 spark_pipeline_framework.transformers.framework_json_to_jsonl_converter.v1.framework_json_to_jsonl_converter spark_pipeline_framework.transformers.framework_kafka_reader spark_pipeline_framework.transformers.framework_kafka_reader.v1 spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader spark_pipeline_framework.transformers.framework_local_file_loader spark_pipeline_framework.transformers.framework_local_file_loader.v1 spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader spark_pipeline_framework.transformers.framework_mapping_runner spark_pipeline_framework.transformers.framework_mapping_runner.v1 spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner_exception spark_pipeline_framework.transformers.framework_parquet_exporter spark_pipeline_framework.transformers.framework_parquet_exporter.v1 spark_pipeline_framework.transformers.framework_parquet_exporter.v1.framework_parquet_exporter spark_pipeline_framework.transformers.framework_parquet_loader spark_pipeline_framework.transformers.framework_parquet_loader.v1 spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader spark_pipeline_framework.transformers.framework_partitioner spark_pipeline_framework.transformers.framework_partitioner.v1 spark_pipeline_framework.transformers.framework_partitioner.v1.framework_partitioner spark_pipeline_framework.transformers.framework_select_columns_transformer spark_pipeline_framework.transformers.framework_select_columns_transformer.v1 spark_pipeline_framework.transformers.framework_select_columns_transformer.v1.framework_select_columns_transformer spark_pipeline_framework.transformers.framework_sql_transformer spark_pipeline_framework.transformers.framework_sql_transformer.v1 spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer spark_pipeline_framework.transformers.framework_transformer spark_pipeline_framework.transformers.framework_transformer.v1 spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer spark_pipeline_framework.transformers.framework_url_loader spark_pipeline_framework.transformers.framework_url_loader.v1 spark_pipeline_framework.transformers.framework_url_loader.v1.framework_url_loader spark_pipeline_framework.transformers.framework_validation_transformer spark_pipeline_framework.transformers.framework_validation_transformer.v1 spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer spark_pipeline_framework.transformers.framework_xml_loader spark_pipeline_framework.transformers.framework_xml_loader.v1 spark_pipeline_framework.transformers.framework_xml_loader.v1.framework_xml_loader spark_pipeline_framework.utilities spark_pipeline_framework.utilities.cannot_cast_exception_parser spark_pipeline_framework.utilities.class_helpers spark_pipeline_framework.utilities.file_downloader spark_pipeline_framework.utilities.file_helpers spark_pipeline_framework.utilities.file_modes spark_pipeline_framework.utilities.flattener spark_pipeline_framework.utilities.FriendlySparkException spark_pipeline_framework.utilities.json_to_jsonl_converter spark_pipeline_framework.utilities.list_utils spark_pipeline_framework.utilities.module_utilities spark_pipeline_framework.utilities.parameter_dict spark_pipeline_framework.utilities.pipeline_helper spark_pipeline_framework.utilities.run_if_else spark_pipeline_framework.utilities.save_csv_to_parquet spark_pipeline_framework.utilities.slack spark_pipeline_framework.utilities.slack.base_slack_client spark_pipeline_framework.utilities.slack.slack_client spark_pipeline_framework.utilities.slack.slack_client_native spark_pipeline_framework.utilities.spark_data_frame_helpers

P

ParameterDict (class in spark_pipeline_framework.utilities.parameter_dict) parameters() (spark_pipeline_framework.pipelines.framework_pipeline.FrameworkPipeline property) (spark_pipeline_framework.pipelines.v2.framework_pipeline.FrameworkPipeline property) parse_cannot_cast_exception() (in module spark_pipeline_framework.utilities.cannot_cast_exception_parser) parse_struct() (in module spark_pipeline_framework.utilities.cannot_cast_exception_parser) pipeline_validation_df_name (in module spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer) post_message_to_slack() (spark_pipeline_framework.utilities.slack.base_slack_client.BaseSlackClient method) (spark_pipeline_framework.utilities.slack.slack_client.SlackClient method) (spark_pipeline_framework.utilities.slack.slack_client_native.SlackClientNative method)

preprocess() (spark_pipeline_framework.transformers.framework_json_loader.v1.framework_json_loader.FrameworkJsonLoader method) (spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader.FrameworkLocalFileLoader method) ProgressLogger (class in spark_pipeline_framework.progress_logger.progress_logger) ProgressLogMetric (class in spark_pipeline_framework.progress_logger.progress_log_metric) ProxyBase (class in spark_pipeline_framework.proxy_generator.proxy_base) ProxyGenerator (class in spark_pipeline_framework.proxy_generator.proxy_generator) PythonProxyBase (class in spark_pipeline_framework.proxy_generator.python_proxy_base)

R

read_file_as_string() (spark_pipeline_framework.proxy_generator.proxy_base.ProxyBase static method) remove_empty_dirs() (spark_pipeline_framework.proxy_generator.proxy_generator.ProxyGenerator static method)

rename_filename_if_exists() (spark_pipeline_framework.utilities.file_downloader.FileDownloader static method) run_id() (spark_pipeline_framework.pipelines.v2.framework_pipeline.FrameworkPipeline property) run_if_else() (in module spark_pipeline_framework.utilities.run_if_else)

S

save_csv_as_parquet() (in module spark_pipeline_framework.utilities.save_csv_to_parquet) sc() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) setMultiLine() (spark_pipeline_framework.transformers.framework_json_loader.v1.framework_json_loader.FrameworkJsonLoader method) setName() (spark_pipeline_framework.proxy_generator.python_proxy_base.PythonProxyBase method) setParams() (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) setProgressLogger() (spark_pipeline_framework.proxy_generator.python_proxy_base.PythonProxyBase method) setStandardParams() (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer method) SlackClient (class in spark_pipeline_framework.utilities.slack.slack_client) SlackClientNative (class in spark_pipeline_framework.utilities.slack.slack_client_native) SlackEventLogger (class in spark_pipeline_framework.event_loggers.slack_logger.v1.slack_event_logger) spark_get_execution_plan() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) spark_is_data_frame_empty() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) spark_pipeline_framework module spark_pipeline_framework.event_loggers module spark_pipeline_framework.event_loggers.event_logger module spark_pipeline_framework.event_loggers.slack_logger module spark_pipeline_framework.event_loggers.slack_logger.v1 module spark_pipeline_framework.event_loggers.slack_logger.v1.slack_event_logger module spark_pipeline_framework.logger module spark_pipeline_framework.logger.yarn_logger module spark_pipeline_framework.pipelines module spark_pipeline_framework.pipelines.framework_param_pipeline module spark_pipeline_framework.pipelines.framework_pipeline module spark_pipeline_framework.pipelines.v2 module spark_pipeline_framework.pipelines.v2.framework_pipeline module spark_pipeline_framework.progress_logger module spark_pipeline_framework.progress_logger.progress_log_metric module spark_pipeline_framework.progress_logger.progress_logger module spark_pipeline_framework.proxy_generator module spark_pipeline_framework.proxy_generator.generate_proxies module spark_pipeline_framework.proxy_generator.proxy_base module spark_pipeline_framework.proxy_generator.proxy_generator module spark_pipeline_framework.proxy_generator.python_proxy_base module spark_pipeline_framework.proxy_generator.python_transformer_helpers module spark_pipeline_framework.proxy_generator.translator_proxy_base module spark_pipeline_framework.transformers module spark_pipeline_framework.transformers.framework_base_exporter module spark_pipeline_framework.transformers.framework_base_exporter.v1 module spark_pipeline_framework.transformers.framework_base_exporter.v1.framework_base_exporter module spark_pipeline_framework.transformers.framework_csv_exporter module spark_pipeline_framework.transformers.framework_csv_exporter.v1 module spark_pipeline_framework.transformers.framework_csv_exporter.v1.framework_csv_exporter module spark_pipeline_framework.transformers.framework_csv_loader module spark_pipeline_framework.transformers.framework_csv_loader.v1 module spark_pipeline_framework.transformers.framework_csv_loader.v1.framework_csv_loader module spark_pipeline_framework.transformers.framework_data_frame_analyzer module spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1 module spark_pipeline_framework.transformers.framework_data_frame_analyzer.v1.framework_data_frame_analyzer module spark_pipeline_framework.transformers.framework_db_query_runner module spark_pipeline_framework.transformers.framework_db_query_runner.v1 module spark_pipeline_framework.transformers.framework_db_query_runner.v1.framework_db_query_runner module spark_pipeline_framework.transformers.framework_drop_duplicates_transformer module spark_pipeline_framework.transformers.framework_drop_duplicates_transformer.v1 module spark_pipeline_framework.transformers.framework_drop_duplicates_transformer.v1.framework_drop_duplicates_transformer module spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer module spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer.v1 module spark_pipeline_framework.transformers.framework_drop_rows_with_null_transformer.v1.framework_drop_rows_with_null_transformer module spark_pipeline_framework.transformers.framework_file_downloader module spark_pipeline_framework.transformers.framework_file_downloader.v1 module spark_pipeline_framework.transformers.framework_file_downloader.v1.framework_file_downloader module spark_pipeline_framework.transformers.framework_fill_na_transformer module spark_pipeline_framework.transformers.framework_fill_na_transformer.v1 module spark_pipeline_framework.transformers.framework_fill_na_transformer.v1.framework_fill_na_transformer module spark_pipeline_framework.transformers.framework_filter_by_column module spark_pipeline_framework.transformers.framework_filter_by_column.v1 module spark_pipeline_framework.transformers.framework_filter_by_column.v1.framework_filter_by_column_transformer module spark_pipeline_framework.transformers.framework_fixed_width_loader module spark_pipeline_framework.transformers.framework_fixed_width_loader.v1 module spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader module spark_pipeline_framework.transformers.framework_if_else_transformer module spark_pipeline_framework.transformers.framework_if_else_transformer.v1 module spark_pipeline_framework.transformers.framework_if_else_transformer.v1.framework_if_else_transformer module spark_pipeline_framework.transformers.framework_jdbc_exporter module spark_pipeline_framework.transformers.framework_jdbc_exporter.v1 module spark_pipeline_framework.transformers.framework_jdbc_exporter.v1.framework_jdbc_exporter module spark_pipeline_framework.transformers.framework_jdbc_reader module spark_pipeline_framework.transformers.framework_jdbc_reader.v1 module spark_pipeline_framework.transformers.framework_jdbc_reader.v1.framework_jdbc_reader module

spark_pipeline_framework.transformers.framework_json_exporter module spark_pipeline_framework.transformers.framework_json_exporter.v1 module spark_pipeline_framework.transformers.framework_json_exporter.v1.framework_json_exporter module spark_pipeline_framework.transformers.framework_json_loader module spark_pipeline_framework.transformers.framework_json_loader.v1 module spark_pipeline_framework.transformers.framework_json_loader.v1.framework_json_loader module spark_pipeline_framework.transformers.framework_json_splitter module spark_pipeline_framework.transformers.framework_json_splitter.v1 module spark_pipeline_framework.transformers.framework_json_splitter.v1.framework_json_splitter module spark_pipeline_framework.transformers.framework_json_to_jsonl_converter module spark_pipeline_framework.transformers.framework_json_to_jsonl_converter.v1 module spark_pipeline_framework.transformers.framework_json_to_jsonl_converter.v1.framework_json_to_jsonl_converter module spark_pipeline_framework.transformers.framework_kafka_reader module spark_pipeline_framework.transformers.framework_kafka_reader.v1 module spark_pipeline_framework.transformers.framework_kafka_reader.v1.framework_kafka_reader module spark_pipeline_framework.transformers.framework_local_file_loader module spark_pipeline_framework.transformers.framework_local_file_loader.v1 module spark_pipeline_framework.transformers.framework_local_file_loader.v1.framework_local_file_loader module spark_pipeline_framework.transformers.framework_mapping_runner module spark_pipeline_framework.transformers.framework_mapping_runner.v1 module spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner module spark_pipeline_framework.transformers.framework_mapping_runner.v1.framework_mapping_runner_exception module spark_pipeline_framework.transformers.framework_parquet_exporter module spark_pipeline_framework.transformers.framework_parquet_exporter.v1 module spark_pipeline_framework.transformers.framework_parquet_exporter.v1.framework_parquet_exporter module spark_pipeline_framework.transformers.framework_parquet_loader module spark_pipeline_framework.transformers.framework_parquet_loader.v1 module spark_pipeline_framework.transformers.framework_parquet_loader.v1.framework_parquet_loader module spark_pipeline_framework.transformers.framework_partitioner module spark_pipeline_framework.transformers.framework_partitioner.v1 module spark_pipeline_framework.transformers.framework_partitioner.v1.framework_partitioner module spark_pipeline_framework.transformers.framework_select_columns_transformer module spark_pipeline_framework.transformers.framework_select_columns_transformer.v1 module spark_pipeline_framework.transformers.framework_select_columns_transformer.v1.framework_select_columns_transformer module spark_pipeline_framework.transformers.framework_sql_transformer module spark_pipeline_framework.transformers.framework_sql_transformer.v1 module spark_pipeline_framework.transformers.framework_sql_transformer.v1.framework_sql_transformer module spark_pipeline_framework.transformers.framework_transformer module spark_pipeline_framework.transformers.framework_transformer.v1 module spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer module spark_pipeline_framework.transformers.framework_url_loader module spark_pipeline_framework.transformers.framework_url_loader.v1 module spark_pipeline_framework.transformers.framework_url_loader.v1.framework_url_loader module spark_pipeline_framework.transformers.framework_validation_transformer module spark_pipeline_framework.transformers.framework_validation_transformer.v1 module spark_pipeline_framework.transformers.framework_validation_transformer.v1.framework_validation_transformer module spark_pipeline_framework.transformers.framework_xml_loader module spark_pipeline_framework.transformers.framework_xml_loader.v1 module spark_pipeline_framework.transformers.framework_xml_loader.v1.framework_xml_loader module spark_pipeline_framework.utilities module spark_pipeline_framework.utilities.cannot_cast_exception_parser module spark_pipeline_framework.utilities.class_helpers module spark_pipeline_framework.utilities.file_downloader module spark_pipeline_framework.utilities.file_helpers module spark_pipeline_framework.utilities.file_modes module spark_pipeline_framework.utilities.flattener module spark_pipeline_framework.utilities.FriendlySparkException module spark_pipeline_framework.utilities.json_to_jsonl_converter module spark_pipeline_framework.utilities.list_utils module spark_pipeline_framework.utilities.module_utilities module spark_pipeline_framework.utilities.parameter_dict module spark_pipeline_framework.utilities.pipeline_helper module spark_pipeline_framework.utilities.run_if_else module spark_pipeline_framework.utilities.save_csv_to_parquet module spark_pipeline_framework.utilities.slack module spark_pipeline_framework.utilities.slack.base_slack_client module spark_pipeline_framework.utilities.slack.slack_client module spark_pipeline_framework.utilities.slack.slack_client_native module spark_pipeline_framework.utilities.spark_data_frame_helpers module spark_table_exists() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) start() (spark_pipeline_framework.progress_logger.progress_log_metric.ProgressLogMetric method) start_mlflow_run() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method) start_pos (spark_pipeline_framework.transformers.framework_fixed_width_loader.v1.framework_fixed_width_loader.ColumnSpec attribute) stop() (spark_pipeline_framework.progress_logger.progress_log_metric.ProgressLogMetric method)

T

ThrowOnErrorOpener (class in spark_pipeline_framework.utilities.file_downloader) to_dicts() (in module spark_pipeline_framework.utilities.spark_data_frame_helpers) transform() (spark_pipeline_framework.pipelines.framework_param_pipeline.FrameworkParamPipeline method)

transformers() (spark_pipeline_framework.proxy_generator.proxy_base.ProxyBase property) (spark_pipeline_framework.transformers.framework_transformer.v1.framework_transformer.FrameworkTransformer property) TranslatorProxyBase (class in spark_pipeline_framework.proxy_generator.translator_proxy_base)

W

write_to_log() (spark_pipeline_framework.progress_logger.progress_logger.ProgressLogger method)

write_transformer() (spark_pipeline_framework.proxy_generator.proxy_generator.ProxyGenerator static method) write_translator() (spark_pipeline_framework.proxy_generator.proxy_generator.ProxyGenerator static method)