whylogs.api.pyspark.experimental.segmented_profiler#

Module Contents#

Functions#

whylogs_pandas_segmented_profiler(...)

column_profile_bytes_aggregator(...)

collect_segmented_column_profile_views(...)

collect_segmented_results(...)

Attributes#

logger

SparkDataFrame

SEGMENT_KEY_FIELD

whylogs.api.pyspark.experimental.segmented_profiler.logger#
whylogs.api.pyspark.experimental.segmented_profiler.SparkDataFrame#
whylogs.api.pyspark.experimental.segmented_profiler.SEGMENT_KEY_FIELD = 'segment_key'#
whylogs.api.pyspark.experimental.segmented_profiler.whylogs_pandas_segmented_profiler(pdf_iterator: Iterable[whylogs.core.stubs.pd.DataFrame], schema: Optional[whylogs.core.DatasetSchema] = None) → Iterable[whylogs.core.stubs.pd.DataFrame]#
Parameters
Return type

Iterable[whylogs.core.stubs.pd.DataFrame]

whylogs.api.pyspark.experimental.segmented_profiler.column_profile_bytes_aggregator(group_by_cols: Tuple[str], profiles_df: whylogs.core.stubs.pd.DataFrame) → whylogs.core.stubs.pd.DataFrame#
Parameters

  • group_by_cols (Tuple[str]) –

  • profiles_df (whylogs.core.stubs.pd.DataFrame) –

Return type

whylogs.core.stubs.pd.DataFrame

whylogs.api.pyspark.experimental.segmented_profiler.collect_segmented_column_profile_views(input_df: pyspark.sql.DataFrame, schema: Optional[whylogs.core.DatasetSchema] = None) → Dict[whylogs.core.segment.Segment, Dict[str, whylogs.core.view.column_profile_view.ColumnProfileView]]#
Parameters
Return type

Dict[whylogs.core.segment.Segment, Dict[str, whylogs.core.view.column_profile_view.ColumnProfileView]]

whylogs.api.pyspark.experimental.segmented_profiler.collect_segmented_results(input_df: pyspark.sql.DataFrame, schema: whylogs.core.DatasetSchema, dataset_timestamp: Optional[datetime.datetime] = None, creation_timestamp: Optional[datetime.datetime] = None) → whylogs.api.logger.result_set.ResultSet#
Parameters
Return type

whylogs.api.logger.result_set.ResultSet