whylogs.api.pyspark.experimental#

Submodules#

Package Contents#

Functions#

collect_column_profile_views(→ Dict[str, ...)

collect_dataset_profile_view(...)

column_profile_bytes_aggregator(...)

whylogs_pandas_map_profiler(...)

collect_segmented_results(...)

whylogs.api.pyspark.experimental.collect_column_profile_views(input_df: pyspark.sql.DataFrame, schema: Optional[whylogs.core.DatasetSchema] = None) → Dict[str, whylogs.core.view.column_profile_view.ColumnProfileView]#
Parameters
Return type

Dict[str, whylogs.core.view.column_profile_view.ColumnProfileView]

whylogs.api.pyspark.experimental.collect_dataset_profile_view(input_df: pyspark.sql.DataFrame, dataset_timestamp: Optional[datetime.datetime] = None, creation_timestamp: Optional[datetime.datetime] = None, schema: Optional[whylogs.core.DatasetSchema] = None) → whylogs.core.view.dataset_profile_view.DatasetProfileView#
Parameters
Return type

whylogs.core.view.dataset_profile_view.DatasetProfileView

whylogs.api.pyspark.experimental.column_profile_bytes_aggregator(group_by_cols: Tuple[str], profiles_df: whylogs.core.stubs.pd.DataFrame) → whylogs.core.stubs.pd.DataFrame#
Parameters

  • group_by_cols (Tuple[str]) –

  • profiles_df (whylogs.core.stubs.pd.DataFrame) –

Return type

whylogs.core.stubs.pd.DataFrame

whylogs.api.pyspark.experimental.whylogs_pandas_map_profiler(pdf_iterator: Iterable[whylogs.core.stubs.pd.DataFrame], schema: Optional[whylogs.core.DatasetSchema] = None) → Iterable[whylogs.core.stubs.pd.DataFrame]#
Parameters
Return type

Iterable[whylogs.core.stubs.pd.DataFrame]

whylogs.api.pyspark.experimental.collect_segmented_results(input_df: pyspark.sql.DataFrame, schema: whylogs.core.DatasetSchema, dataset_timestamp: Optional[datetime.datetime] = None, creation_timestamp: Optional[datetime.datetime] = None) → whylogs.api.logger.result_set.ResultSet#
Parameters
Return type

whylogs.api.logger.result_set.ResultSet